package cn.cup.dmp.report

import java.util.Properties

import cn.cup.dmp.config.ConfigHelper
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.{DataFrame, SQLContext}

object Province_City_Count {
  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setAppName("DMP平台").setMaster("local[*]")
    conf.set("spark.serializer",ConfigHelper.ser)//设置序列化方式
    val sc = new SparkContext(conf)
    val sql = new SQLContext(sc)

    //读取parquet文件
    val dataFrame: DataFrame = sql.read.parquet(ConfigHelper. destPath)
    //注册临时表
   dataFrame.registerTempTable("t_cleaned_logs")

    //sql查询 统计各省市数据分布情况
    val frame: DataFrame = sql.sql("select provincename,cityname,count(*) ct from t_cleaned_logs group by provincename,cityname")
    //frame.coalesce(1)
    frame.coalesce(1).write.json("D:\\\\小牛视频\\\\资料_广告项目\\省市统计数量报表.txt")

    val connetProperties = new Properties()
    connetProperties.setProperty("user","root")
    connetProperties.setProperty("password","123456")
    frame.write.mode("overwrite").jdbc(ConfigHelper.url,ConfigHelper.tableName,connetProperties)

    sc.stop()

  }

}
